{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "43868569",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c4b3f9aa",
   "metadata": {},
   "source": [
    "### 标准卷积"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "004e972e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 3, 32, 36])\n",
      "torch.Size([4, 16, 32, 36])\n"
     ]
    }
   ],
   "source": [
    "\n",
    "class Conv(nn.Module):\n",
    "    \n",
    "    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, padding=1):\n",
    "        super().__init__()\n",
    "        self.conv = nn.Conv2d(\n",
    "            in_channels=in_channels,\n",
    "            out_channels=out_channels,\n",
    "            kernel_size=kernel_size,\n",
    "            stride=stride,\n",
    "            padding=padding,\n",
    "            bias=True,\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.conv(x)\n",
    "\n",
    "x = torch.randn(4, 3, 32, 36)\n",
    "net = Conv(in_channels=3, out_channels=16)\n",
    "out = net(x)\n",
    "\n",
    "print(x.shape)\n",
    "\n",
    "print(out.shape)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1e20678e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 3, 32, 32])\n",
      "torch.Size([4, 24, 32, 32])\n"
     ]
    }
   ],
   "source": [
    "class DilationConv(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels, kernel_size=3, stride=1, dilation=2):\n",
    "        \n",
    "        super().__init__()\n",
    "        padding = (kernel_size - 1) * dilation // 2\n",
    "\n",
    "        self.conv = nn.Conv2d(\n",
    "            in_channels=in_channels,\n",
    "            out_channels=out_channels,\n",
    "            kernel_size=kernel_size,\n",
    "            stride=stride,\n",
    "            padding=padding,\n",
    "            dilation=dilation,\n",
    "            bias=True\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.conv(x)\n",
    "    \n",
    "\n",
    "x = torch.randn(4, 3, 32, 32)\n",
    "\n",
    "net = DilationConv(in_channels=3, out_channels=24)\n",
    "\n",
    "y = net(x)\n",
    "print(x.shape)\n",
    "print(y.shape)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "622dfaad",
   "metadata": {},
   "source": [
    "## 池化"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22342442",
   "metadata": {},
   "source": [
    "### 最大池化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5dbec3a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 3, 16, 16])\n"
     ]
    }
   ],
   "source": [
    "max_pool = nn.MaxPool2d(\n",
    "    kernel_size=2,\n",
    "    stride=2,\n",
    "    padding=0\n",
    "    )\n",
    "\n",
    "\n",
    "x = torch.randn(1, 3, 32, 32)\n",
    "y = max_pool(x)\n",
    "print(y.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5c4863a0",
   "metadata": {},
   "source": [
    "### 平均池化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b06a02d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 3, 32, 32])\n"
     ]
    }
   ],
   "source": [
    "avg_pool = nn.AvgPool2d(kernel_size=2, stride=2)\n",
    "# print(x)\n",
    "y = avg_pool(x)\n",
    "print(y.shape)\n",
    "# y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9cd4c578",
   "metadata": {},
   "source": [
    "### 全局平局池化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "566d3d6d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 3, 1, 1])\n",
      "tensor([[[[-0.0163]],\n",
      "\n",
      "         [[-0.0306]],\n",
      "\n",
      "         [[-0.0433]]]])\n"
     ]
    }
   ],
   "source": [
    "# 设置输出形状\n",
    "# 只有设置为1才是全局池化\n",
    "global_avg_pool = nn.AdaptiveAvgPool2d(1)\n",
    "y = global_avg_pool(x)\n",
    "print(y.shape)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "9fa33ecd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 3, 32, 32])\n"
     ]
    }
   ],
   "source": [
    "# 自适应池化\n",
    "adaptive_pool = nn.AdaptiveAvgPool2d((7,7))\n",
    "y = adaptive_pool(x)\n",
    "print(x.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e402fd01",
   "metadata": {},
   "source": [
    "## 激活函数"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c3d0bbf",
   "metadata": {},
   "source": [
    "## ReLU\n",
    "$$\n",
    "\\max{(0, x)} \\in [0, + \\inf )\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "19ebbba7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[ 1.1330],\n",
      "          [ 0.3127],\n",
      "          [-1.0659],\n",
      "          [-0.5526],\n",
      "          [-1.4990],\n",
      "          [-0.6312],\n",
      "          [-1.5851],\n",
      "          [-0.8705],\n",
      "          [-0.1024],\n",
      "          [ 1.2696]]]])\n",
      "tensor([[[[1.1330],\n",
      "          [0.3127],\n",
      "          [0.0000],\n",
      "          [0.0000],\n",
      "          [0.0000],\n",
      "          [0.0000],\n",
      "          [0.0000],\n",
      "          [0.0000],\n",
      "          [0.0000],\n",
      "          [1.2696]]]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(1,1,10,1)\n",
    "print(x)\n",
    "relu = nn.ReLU()\n",
    "y = relu(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18783876",
   "metadata": {},
   "source": [
    "### Leaky ReLU\n",
    "$$\n",
    "\\max{(\\alpha x, x)} \\in (-\\inf, +\\inf) \\\\\n",
    "需要调整\\alpha ， 默认 \\alpha = 0.01\n",
    "$$\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1e817a33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[ 1.2237],\n",
      "          [ 0.5003],\n",
      "          [ 0.1129],\n",
      "          [-0.5953],\n",
      "          [ 1.1839]]]])\n",
      "tensor([[[[ 1.2237],\n",
      "          [ 0.5003],\n",
      "          [ 0.1129],\n",
      "          [-0.0060],\n",
      "          [ 1.1839]]]])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(1, 1, 5, 1)\n",
    "leaky_relu = nn.LeakyReLU()\n",
    "y = leaky_relu(x)\n",
    "print(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "11647357",
   "metadata": {},
   "source": [
    "### PReLU\n",
    "$$\n",
    "\\max(\\alpha x, x) ，其中\\alpha 可以自动学习\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a83ecdf8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[ 1.2237],\n",
      "          [ 0.5003],\n",
      "          [ 0.1129],\n",
      "          [-0.1488],\n",
      "          [ 1.1839]]]], grad_fn=<PreluKernelBackward0>)\n"
     ]
    }
   ],
   "source": [
    "prelu = nn.PReLU()\n",
    "y = prelu(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5f61b27",
   "metadata": {},
   "source": [
    "### ELU\n",
    "$$\n",
    "x \\ if  \\ x > 0 \\ else \\ \\alpha(e^x - 1) \\ \\in (-\\alpha, +\\inf)\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f4da4a88",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[ 1.2237],\n",
      "          [ 0.5003],\n",
      "          [ 0.1129],\n",
      "          [-0.4486],\n",
      "          [ 1.1839]]]])\n"
     ]
    }
   ],
   "source": [
    "elu = nn.ELU()\n",
    "y = elu(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "167dc581",
   "metadata": {},
   "source": [
    "### Sigmoid\n",
    "$$\n",
    "Sigmoid = \\frac{1}{1+e^{-x}} \\in (0, 1)\n",
    "$$\n",
    "平滑，可解释概率"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2c3de5d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[0.7727],\n",
      "          [0.6225],\n",
      "          [0.5282],\n",
      "          [0.3554],\n",
      "          [0.7656]]]])\n"
     ]
    }
   ],
   "source": [
    "sigmoid = nn.Sigmoid()\n",
    "y = sigmoid(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b211e4d",
   "metadata": {},
   "source": [
    "### Tanh\n",
    "$$\n",
    "\n",
    "Tanh = \\frac{e^x - e^{-x}}{e^x + e^{-x}} \\in (-1,1)\n",
    "$$\n",
    "零均值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "505efcca",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[ 0.8408],\n",
      "          [ 0.4624],\n",
      "          [ 0.1124],\n",
      "          [-0.5337],\n",
      "          [ 0.8287]]]])\n"
     ]
    }
   ],
   "source": [
    "tanh = nn.Tanh()\n",
    "y = tanh(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cb50de10",
   "metadata": {},
   "source": [
    "### Swish\n",
    "$$\n",
    "x \\theta(\\beta x)\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1b4b652c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[ 0.9456],\n",
      "          [ 0.3115],\n",
      "          [ 0.0596],\n",
      "          [-0.2116],\n",
      "          [ 0.9064]]]])\n"
     ]
    }
   ],
   "source": [
    "silu = nn.SiLU()\n",
    "y = silu(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78066532",
   "metadata": {},
   "source": [
    "### GELU\n",
    "$$\n",
    "x \\cdot \\Phi (x) \\in (-\\inf, +\\inf)\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7816c5f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[ 1.0885],\n",
      "          [ 0.3460],\n",
      "          [ 0.0615],\n",
      "          [-0.1642],\n",
      "          [ 1.0439]]]])\n"
     ]
    }
   ],
   "source": [
    "gelu = nn.GELU()\n",
    "y = gelu(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d6061ee",
   "metadata": {},
   "source": [
    "### Softmax\n",
    "$$\n",
    "\\frac{e^{x_i}}{\\sum_i e^{x_i}} \\in (0, 1) \\ 和为1\n",
    "\n",
    "$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "491c6a49",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[[1.],\n",
      "          [1.],\n",
      "          [1.],\n",
      "          [1.],\n",
      "          [1.]]]])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\xjtu\\AppData\\Local\\Temp\\ipykernel_3716\\3859895188.py:2: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  y = F.softmax(x)\n"
     ]
    }
   ],
   "source": [
    "# softmax = F.softmax()\n",
    "y = F.softmax(x)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "40a4fafe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 3, 16, 16])\n"
     ]
    }
   ],
   "source": [
    "# 计算每个通道的均值、方差\n",
    "x = torch.randn(4,3,16,16)\n",
    "batchnorm = nn.BatchNorm2d(num_features=3)\n",
    "y = batchnorm(x)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "88193227",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 3, 16, 16])\n"
     ]
    }
   ],
   "source": [
    "layer = nn.LayerNorm(normalized_shape=(3, 16, 16))\n",
    "y = layer(x)\n",
    "print(y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "19d8fce7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([4, 3, 16, 16])\n"
     ]
    }
   ],
   "source": [
    "instanceNorm = nn.InstanceNorm2d(num_features=3, affine=True)\n",
    "y = instanceNorm(x)\n",
    "print(y.shape)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
